# comun self hosted implementation: tokenizer library
# module prefix: t
#
# This module contains resources that will help parsing comun source code as
# a stream of language tokens.

~"cmn_general.cmn"

~_CMN_tState
~_CMN_tSTBackup:0   # ST backup

CMN_tTOKEN_NONE:     0   . # no token detected
CMN_tTOKEN_COMMAND:  1   . # command such as "<=", ".", "+", "$>abc", ...
CMN_tTOKEN_NAME:     2   . # name of a func (call)
CMN_tTOKEN_FUNC_DEF: 3   . # name of a func with ":" appended
CMN_tTOKEN_NUMBER:   4   . # numeric literal
CMN_tTOKEN_STRING:   5   . # string literal
CMN_tTOKEN_ERROR:    255 . # badly formed token

_CMN_tSTATE_BLANK:   0   . # reading blank chars between tokens
_CMN_tSTATE_COMMENT: 1   . # inside comment
_CMN_tSTATE_STRING:  2   . # inside string literal
_CMN_tSTATE_NAME:    3   . # reading name (identifier)
_CMN_tSTATE_FUNDEF:  4   . # just read func definition
_CMN_tSTATE_SIGN:    5   . # starting + or -
_CMN_tSTATE_NUMBER:  6   . # num, next must be digit or end
_CMN_tSTATE_COMMAND: 7   . # reading command

CMN_tCharIsBlank: " " <= .  # TODO: handle also square brackets

CMN_tCharIsNameChar: 
  $0 "0" >= $1 "9" <= &&
  $1 "a" >= $2 "z" <= && ||
  $1 "A" >= $2 "Z" <= && ||
  >< "_" = ||
.

# arg: t0 t1 ... tn
# ret: t0 t1 ... tn 0 tn ... t1 t0
# Converts a token from tokenizer format to a normal reversed zero ter. str.
_CMN_tTokenToStr:
  $0>_CMN_gP1
  0
  1 @
    $_CMN_gP1
    $<_CMN_gP1
    $_CMN_gP1=_CMN_tSTBackup
  .
.

CMN_tCharIsHexDig:
  $0 "0" >= $1 "9" <= &&
  $1 "a" >= $2 "f" <= && ||
  >< ^
.

CMN_tCharIsCommChar:
  $0 CMN_tCharIsBlank !!
  $1 "#" != &&
  >< 34 != && # '"'
.

# Initializes tokenizer, has to be called before tokenizer is used.
CMN_tInit:
  $0>_CMN_tSTBackup
  _CMN_tSTATE_BLANK $:_CMN_tState
.

CMN_tPopToken:
  $_CMN_tSTBackup>0
.

# arg: token pattern
# ret: bool
# Checks if currently read token matches given pattern. The token arg is
# currently read token (NOT a zero terminated string), pattern is either 0 or
# 1 terminated string, pushed backwards (just like normal comun strings);
# terminating 0 means the pattern has to match exactly, terminating 1 means
# the pattern may match only up to this val. The pattern arg will be popped,
# token won't.
CMN_tTokenMatch:
  1 $:_CMN_gV1 # matches?

  $_CMN_tSTBackup>_CMN_gP1
  $>_CMN_gP1

  @@
    $0 1 = ?
      ^
      !@
    .

    $0 $_CMN_gP1 != ?
      0 $:_CMN_gV1
    .

    !! ?
      !@
    .

    $>_CMN_gP1
  .

  $_CMN_gV1
.

# arg: token
# ret: token state 3val
# Extracts val from a token with numeric literal, expects token on input (in
# tokenizer's format, NOT a normal zero terminated string); this token will
# remain unpopped. Note the token may be a numeric literal but also another
# token that may contain such literal (e.g. "~myvar:100"). The val is returned
# as 32 bit integer in TE 32. Returns state, which may be: 0 (error), 1
# (positive) or 2 (negative, i.e. numeric literal was prepended with '-').
CMN_tLiteralVal:
   $0>_CMN_gP1  

   0 # stopper

   @@ # go from right to the left, find start
     $_CMN_gP1 "+" != $_CMN_gP1 "-" != &
     $_CMN_gP1 CMN_tCharIsHexDig !! &
     $_CMN_gP1 "x" != &
     $0=_CMN_gP1 !! | ?
       !@
     .

     $<_CMN_gP1
   .

   $>_CMN_gP1  

   0 # negative ?

   $_CMN_gP1 "+" = ?
     $>_CMN_gP1  
   ; $_CMN_gP1 "-" = ?
     ^ 1 $>_CMN_gP1
   . .

   10 # base

   $_CMN_gP1 "d" = ?
     $>_CMN_gP1  
   ; $_CMN_gP1 "x" = ?
     ^ 16 $>_CMN_gP1
   ; $_CMN_gP1 "b" = ?
     ^ 2 $>_CMN_gP1
   . . .

   ~32 0 ~0

   @@
     $_CMN_gP1 !! ?
       !@
     .

     $_CMN_gP1 # digit val

     $0 $0 "0" >= >< "9" <= &
       "0"
       "a" 10 - ??
     -

     $0 $2 >= ? # digit >= base => ERROR
       ^ ^ 0
       !.
     .

     ~32 0 ~0 $1 >32 ~32 * ~0
     ~32 0 ~0 >32    ~32 + ~0

     $>_CMN_gP1  
   .

   ^ # base

   ? # nagative ?
     2 ~32 -1 * ~0
   ;
     1
   .

   >< ^ # pop stopper 0, leave only result  
.

# arg: char
# ret: tokenType
# Feeds the next char to tokenizer, which may or may NOT be popped by this func,
# which the caller must leave so -- this behavior is such so that once the
# complete token has been read and detected, it will remain on the stack so that
# it can be analyzed (after which the whole token can be popped with
# CMN_tPopToken). Pushes the type of token detected; this val on the other hand
# MUST be popped by the caller before feeding another char. Feeding 0 means end
# of input.
CMN_tFeedChar:
  # TODO: order branches by probability?
  # BUT WATCH OUT, some order has to be kept (e.g. check digit before name!)

  $_CMN_tState _CMN_tSTATE_COMMENT = ?
    $0 "#" = >< 10 = || ?     # comment end?
      _CMN_tSTATE_BLANK $:_CMN_tState
    .
    # no pop is here, it was done by the above branch

    CMN_tTOKEN_NONE
  ;
  $_CMN_tState _CMN_tSTATE_STRING = ?
    $0 34 = ?    # '"' ?
      _CMN_tSTATE_BLANK $:_CMN_tState
      CMN_tTOKEN_STRING
    ;
      CMN_tTOKEN_NONE
    .
  ;
  $_CMN_tState _CMN_tSTATE_COMMAND = ?
    $0 CMN_tCharIsBlank ?
      ^
      _CMN_tSTATE_BLANK $:_CMN_tState
      CMN_tTOKEN_COMMAND
    ;
    $0 "#" = ?
      ^
      _CMN_tSTATE_COMMENT $:_CMN_tState
      CMN_tTOKEN_COMMAND
    ;
      _CMN_tSTATE_COMMAND $:_CMN_tState
      CMN_tTOKEN_NONE
    . .
  ;
  $_CMN_tState _CMN_tSTATE_NAME = ?
    $0 ":" = ?
      _CMN_tSTATE_FUNDEF $:_CMN_tState
      CMN_tTOKEN_NONE
    ;
    $0 CMN_tCharIsBlank ?
      ^
      _CMN_tSTATE_BLANK $:_CMN_tState
      CMN_tTOKEN_NAME
    ;
    $0 "#" = ?
      ^
      _CMN_tSTATE_COMMENT $:_CMN_tState
      CMN_tTOKEN_NAME
    ;
    $0 CMN_tCharIsNameChar ?
      CMN_tTOKEN_NONE
    ;
      _CMN_tSTATE_COMMAND $:_CMN_tState
      CMN_tTOKEN_NONE
    . . . .
  ;
  $_CMN_tState _CMN_tSTATE_FUNDEF = ?
    # expecting end after ':'
    $0 CMN_tCharIsBlank ?
      ^
      _CMN_tSTATE_BLANK $:_CMN_tState
      CMN_tTOKEN_FUNC_DEF
    ;
    $0 "#" = ?
      ^
      _CMN_tSTATE_COMMENT $:_CMN_tState
      CMN_tTOKEN_FUNC_DEF
    ;
      CMN_tTOKEN_ERROR
    . .
  ;
  $_CMN_tState _CMN_tSTATE_SIGN = ?
    $0 "d" = $1 "x" = || $1 "b" = || $1 CMN_tCharIsHexDig || ?
      _CMN_tSTATE_NUMBER $:_CMN_tState
      CMN_tTOKEN_NONE
    ;
    $0 "#" = ?
      ^
      _CMN_tSTATE_COMMENT $:_CMN_tState
      CMN_tTOKEN_COMMAND
    ;
    $0 CMN_tCharIsBlank ?
      ^
      _CMN_tSTATE_BLANK $:_CMN_tState
      CMN_tTOKEN_COMMAND
    ;
      _CMN_tSTATE_COMMAND $:_CMN_tState
      CMN_tTOKEN_NONE
    . . .
  ;
  $_CMN_tState _CMN_tSTATE_NUMBER = ?
    $0 CMN_tCharIsHexDig ?
      CMN_tTOKEN_NONE
    ;
    $0 CMN_tCharIsBlank ?
      ^
      _CMN_tSTATE_BLANK $:_CMN_tState
      CMN_tTOKEN_NUMBER
    ;
    $0 "#" = ?
      ^
      _CMN_tSTATE_COMMENT $:_CMN_tState
      CMN_tTOKEN_NUMBER
    ;
      CMN_tTOKEN_ERROR
    . . .
  ;
  # _CMN_tSTATE_BLANK
    $0 "#" = ?                  # comment start?
      ^
      _CMN_tSTATE_COMMENT $:_CMN_tState
    ;
    $0 $0 "+" = >< "-" = || ?   # sign?
      _CMN_tSTATE_SIGN $:_CMN_tState
    ;
    $0 $0 "0" >= >< "9" <= && ? # decimal digit?
      _CMN_tSTATE_NUMBER $:_CMN_tState
    ;
    $0 34 = ?                   # '"', string start?
      _CMN_tSTATE_STRING $:_CMN_tState
    ;
    $0 CMN_tCharIsNameChar ?
      _CMN_tSTATE_NAME $:_CMN_tState
    ;
    $0 CMN_tCharIsBlank !! ?    # non-blank?
      _CMN_tSTATE_COMMAND $:_CMN_tState
    ;
      ^
    . . . . . .
 
    CMN_tTOKEN_NONE
  . . . . . . .
.
